package com.aerfafish.spider.spider;

import com.aerfafish.spider.callback.SpiderCallback;
import com.aerfafish.spider.constant.SpiderConfig;
import com.aerfafish.spider.exception.NoFundException;
import com.aerfafish.spider.neo4j.dao.Date;
import com.aerfafish.spider.neo4j.dao.Fund;
import com.aerfafish.spider.neo4j.dao.FundHoldShare;
import com.aerfafish.spider.neo4j.dao.FundValue;
import com.aerfafish.spider.spider.download.CustomHttpClientDownloader;
import com.aerfafish.spider.utils.RequestUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.GregorianCalendar;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author aerfafish
 * @date 2021/4/12 2:16 下午
 */
public class FundSpider implements PageProcessor {

    private final static Logger logger = LoggerFactory.getLogger(FundSpider.class);

    private Spider spider = Spider.create(this).setExitWhenComplete(false).thread(10).setDownloader(new CustomHttpClientDownloader());

    private SpiderCallback spiderCallback;

    private String currentDay = null;

    private String lastDay = null;

    public FundSpider(SpiderCallback spiderCallback) {
        java.util.Date currentTime = new java.util.Date();
        SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
        currentDay = formatter.format(currentTime);
        Calendar calendar = new GregorianCalendar();
        calendar.setTime(currentTime);
        calendar.add(Calendar.DATE,-1);
        lastDay= formatter.format(calendar.getTime());

        this.spiderCallback = spiderCallback;
        spider.start();
    }



    @Override
    public void process(Page page) {

        String fundCode = null;

        if (page.getUrl().regex("^http://api\\.fund\\.eastmoney\\.com/f10/lsjz\\?.*").match()) {
            Pattern fundCodePattern = Pattern.compile("(fundCode=)([0-9]*)");
            Matcher matcher = fundCodePattern.matcher(page.getUrl().get());
            try {
                if (matcher.find()) {
                    fundCode = matcher.group(2);
                } else {
                    logger.error("fundcode not match");
                    return;
                }

                if (fundCode == null) {
                    logger.error("fund code is null.");
                    return;
                };
            } catch (Exception e) {
                logger.error(page.getUrl().get(), e);
                return;
            }

            // 解析基金净值
            parseFundValue(fundCode, page.getRawText());

            Calendar cal = Calendar.getInstance();
            int year = cal.get(Calendar.YEAR);
            int month = cal.get(Calendar.MONTH )+1;
            if (month <= 3) {
                // 爬取上一年数据
                spiderShareInfo(fundCode, year-1, 4);
            } else if (month <= 6){
                spiderShareInfo(fundCode, year, 1);
            } else if (month <= 9){
                spiderShareInfo(fundCode, year, 2);
            } else {
                spiderShareInfo(fundCode, year, 3);
            }

        } else if (page.getUrl()
                .regex("^http://fundf10\\.eastmoney\\.com/FundArchivesDatas\\.aspx\\?type=jjcc.*").match()) {
            Integer quarter = null;
            Pattern fundCodePattern = Pattern.compile("(code=)([0-9]*)");
            Pattern quarterPattern = Pattern.compile("(month=)([0-9]*)");
            Matcher fundCodeMatcher = fundCodePattern.matcher(page.getUrl().get());
            Matcher quarterMatcher = quarterPattern.matcher(page.getUrl().get());

            try {
                if (fundCodeMatcher.find()) {
                    fundCode = fundCodeMatcher.group(2);
                } else {
                    logger.error("fundcode not match");
                    return;
                }

                if (quarterMatcher.find()) {
                    quarter = Integer.parseInt(quarterMatcher.group(2))/3;
                } else {
                    logger.error("quarter not match");
                    return;
                }

                if (fundCode == null) {
                    logger.error("fund code is null.");
                    return;
                }
            } catch (Exception e) {
                logger.error(page.getUrl().get(), e);
                return;
            }

            parseShareInfo(fundCode, page.getRawText(), quarter);

        } else if ((page.getUrl().regex("^http://fundf10\\.eastmoney\\.com/jjjl_").match())) {
            Pattern fundCodePattern = Pattern.compile("(jjjl_)([0-9]*)");
            Matcher fundCodeMatcher = fundCodePattern.matcher(page.getUrl().get());
            try {
                if (fundCodeMatcher.find()) {
                    fundCode = fundCodeMatcher.group(2);
                } else {
                    logger.error("fundcode not match");
                    return;
                }

                if (fundCode == null) {
                    logger.error("fund code is null.");
                    return;
                }

            } catch (Exception e) {
                logger.error(page.getUrl().get(), e);
                return;
            }
            try {
                parseCompanyInfo(fundCode, page.getHtml());
                spiderHistoryValues(fundCode);
            } catch (NoFundException e) {
                logger.info(e.getMessage());
            } catch (Exception e) {
                // 存在部分后端和前端code不同例如000002和000001
                logger.error("", e);
            }
        } else {
            logger.error(page.getUrl().get());
        }
    }

    @Override
    public Site getSite() {
        return Site.me().setCycleRetryTimes(3000).setSleepTime(10).setCharset("UTF-8").setTimeOut(30000)
                .setUserAgent("Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " +
                        "Chrome/70.0.3538.77 Safari/537.36");
    }

    public void spiderFund(String fundCode) {

        if (fundCode == null) {
            logger.error("fundCode is null");
            return;
        }

        spiderCompany(fundCode);


        // 季度股票投资明细 http://fundf10.eastmoney.com/ccmx_002190.html
//        String url = "http://fundf10.eastmoney.com/FundArchivesDatas.aspx?type=jjcc&code=002190&topline=10&year=&month=12&rt=0.04566403789215134";

        // 基金经理 基金公司
//        String url = "http://fundf10.eastmoney.com/jjjl_002190.html";

        // 基金经理当前管理基金
        // "var mgrlists=[{valueField:'006977',name:'农银汇理海棠三年定开混合',type:'2'},{valueField:'001606',name:'农银工业4.0混合',type:'2'},{valueField:'002190',name:'农银新能源主题',type:'2'},{valueField:'000336',name:'农银研究精选混合',type:'2'}];"


        // 基金公司当前股票
        // http://fund.eastmoney.com/company/80092233.html
        // var gpccGpdms = "0.300750,0.002460,1.601012,0.000733,1.600438,0.002709,0.300037,0.300073,1.603659,0.000858"; var gpccLimitDate = "2020-12-31"
        // 具体持仓信息
        // http://fund.eastmoney.com/Company/tzzh/GsccQuarter?gsId=80092233&year=2020&quarter=4&ftype=0

        // 阶段涨幅
//        String url = "http://fundf10.eastmoney.com/FundArchivesDatas.aspx?type=jdzf&code=002190&rt=0.05288963827472126"；

        // 净值 http://fundf10.eastmoney.com/jjjz_002190.html FSRQ净值日期 DWJZ单位净值 LJJZ累计净值 JZZZL日增长率 SGZT申购状态 SHZT赎回状态

    }

    private void spiderHistoryValues(String fundCode) {
        String url = "http://api.fund.eastmoney.com/f10/lsjz" +
                "?callback=%s&fundCode=%s&pageIndex=%s&pageSize=20&startDate=&endDate=&_=%s";

        for (int i = 1; i <= SpiderConfig.pageNum; i++) {
            spider.addRequest(RequestUtil.urlToRequest(
                    String.format(url, "Data", fundCode, 1, System.currentTimeMillis()),
                    "http://fundf10.eastmoney.com"));
        }
    }

    private void spiderShareInfo(String fundCode, Integer year, Integer quarter) {
        // 第四季度持股情况
        String url = "http://fundf10.eastmoney.com/FundArchivesDatas.aspx" +
                "?type=jjcc&code=%s&topline=10&year=%s&month=%s&rt=0.04566403789215134";

        spider.addRequest(RequestUtil.urlToRequest(
                String.format(url, fundCode, year, quarter*3),
                "http://fundf10.eastmoney.com"));
    }

    private void spiderCompany(String fundCode) {
        // 爬取基金管理人和基金公司情况
        String url = "http://fundf10.eastmoney.com/jjjl_%s.html";

        spider.addRequest(
                RequestUtil.urlToRequest(String.format(url, fundCode),
                        "http://fundf10.eastmoney.com"));
    }

    private void parseFundValue(String fundCode, String data) {
        String parsedData = null;
        Pattern r = Pattern.compile("(\"LSJZList\":\\[)([^]]*)(])");
        Matcher matcher = r.matcher(data);
        try {
            if(matcher.find()) {
                parsedData = matcher.group(2);
                if (parsedData == null) {
                    throw new Exception("parseData not match");
                }
            } else {
                logger.error("parse fund value error");
                logger.error(data);
                return;
            }
        } catch (Exception e) {
            logger.error("parse fund value to string error", e);
            return;
        }
        if ("".equals(parsedData)) {
            logger.info("fundCode: " + fundCode + " fundValue is null, maybe 当前产品为QDII基金，投资于境外市场，暂无净值估算数据。");
            return;
        }
        String[] fundValues = parsedData.split("(?<=}),(?=\\{)");
        for (String fundString : fundValues) {
            FundValue fundValue = new FundValue();
            try {

                JSONObject fundObject = JSON.parseObject(fundString);
                String dateString = fundObject.getString("FSRQ");
                if (dateString == null || dateString.equals("") || (!dateString.equals(currentDay) && !dateString.equals(lastDay))) {
                    continue;
                }
                Date date = new Date();
                date.setDate(dateString);
                fundValue.setDate(date);
                fundValue.setAverageValue(Float.valueOf(fundObject.getString("DWJZ") == null ? fundObject.getString("DWJZ"):"0"));
                fundValue.setAccumulatedValue(Float.valueOf(fundObject.getString("LJJZ") == null ? fundObject.getString("LJJZ"):"0"));
                fundValue.setDailyGrowthRate(Float.valueOf(fundObject.getString("JZZZL") == null ? fundObject.getString("JZZZL"):"0"));
                fundValue.setRedemptionStatus(fundObject.getString("SHZT") == null ? fundObject.getString("SHZT"):"");
                fundValue.setSubscriptionStatus(fundObject.getString("SGZT") == null ? fundObject.getString("SGZT"):"");
                spiderCallback.onSaveFundDateRelation(fundValue, fundCode, date.getDate());
            } catch (Exception e) {
                logger.error("parse fund value to json error, fundCode: " + fundCode, e);
                return;
            }
        }
    }

    private void parseShareInfo(String fundCode, String data, Integer quarter) {
        String parsedData = null;
        Pattern r = Pattern.compile("(content:\")(.*)(\")");
        Matcher matcher = r.matcher(data);
        Fund fund = new Fund();
        Boolean isUpdateFund = false;
        fund.setCode(fundCode);
        try {
            if (matcher.find()) {
                parsedData = matcher.group(2);
            }
        } catch (Exception e) {
            logger.error(data, e);
        }

        Html html = Html.create(parsedData);
        Selectable selectable = html.$("div.boxitem.w790");
        String s = selectable.toString();

        for (Selectable node : selectable.nodes()) {
            if (node.toString().contains(quarter.toString() + "季度股票投资明细")) {
                Selectable shareDatas = node.$("tbody").$("tr");
                for (Selectable shareData : shareDatas.nodes()) {
                    FundHoldShare fundHoldShare = new FundHoldShare();
                    try {
                        String shareName = shareData.$("td.tol").$("a").regex("(>)(.*)(<)", 2).toString();
                        String shareCode = shareData.$("td.tol").links()
                                .regex("(^//quote.eastmoney.com/)(.*)(.html$)", 2).toString();
                        List<String> detailed = shareData.$("td.tor").regex(">(((?!span).)+)<", 1).all();
                        if (detailed.size() < 3) {
                            // 美股
                            shareName = shareData.$("td.toc").regex("(<span>)(.*)(</span>)", 2).toString();
                            List<String> shareCodeList = shareData.$("td.toc").links()
                                    .regex("(^//quote.eastmoney.com/)(.*)(.html$)", 2).all();
                            if (shareCodeList.size() == 0) {
                                logger.info("shareName: " + shareName + " shareCode is null fundCode: " + fundCode);
                                return;
                            }
                            shareCode = shareCodeList.get(0);
                            detailed = shareData.$("td.toc").regex(">(((?!span|a).)+)</td>", 1).all();
                            if (detailed.size() < 3) {
                                logger.error("parse share info error " + shareData + " fundCode: " + fundCode);
                                return;
                            }

                        }
                        try {
                            fundHoldShare.setProportion(
                                    Float.parseFloat(detailed.get(0).substring(0, detailed.get(0).length()-1).replace(",", "")));
                            fundHoldShare.setSharesNum(
                                    Float.parseFloat(detailed.get(1).replace(",", "")));
                            fundHoldShare.setSharesValue(
                                    Float.parseFloat(detailed.get(2).replace(",", "")));
                            if (!isUpdateFund) {
                                float asset = fundHoldShare.getSharesValue() * 100 / fundHoldShare.getProportion();
                                if (fund.getAssetSize() == null) {
                                    fund.setAssetSize((long) asset);
                                } else {
                                    if ((fund.getAssetSize()/asset) > 2 || (fund.getAssetSize()/asset) < 0.5) {
                                        // 说明净值数据有错误
                                        fund.setAssetSize((long) asset);
                                    } else {
                                        isUpdateFund = true;
                                    }
                                }
                            }

                        } catch (IndexOutOfBoundsException e) {
                            logger.error("parse share info error " + " fundCode: " + fundCode, detailed);
                            return;
                        }
                        spiderCallback.onSaveFundShareRelation(fundHoldShare, fundCode, shareCode);
                    } catch (Exception e) {
                        logger.error("parse share info error" + " fundCode: " + fundCode, e);
                    }
                }
                break;
            }
        }
        try {
            spiderCallback.onSaveFund(fund);
        } catch (Exception e) {
            logger.error("", e);
        }

    }

    private void parseCompanyInfo(String fundCode, Html data) throws Exception {
        Fund fund = null;
        fund = new Fund();
        fund.setCode(fundCode);
        String companyCode = null;
        String managerCode = null;
        String establishDate = null;
        List<Selectable> fundDatas = null;
        try {
            fundDatas = data.$("div.bs_gl").$("label").nodes();
            for (Selectable fundData : fundDatas) {
                if (fundData.toString().contains("成立日期")) {
                    establishDate =  fundData.$("span").regex("(>)(.*)(<)", 2).toString();
                } else if (fundData.toString().contains("基金经理")) {
                    managerCode =  fundData.links()
                            .regex("(^http://fund.eastmoney.com/manager/)([0-9]*)(.html$)", 2).toString();
                } else if (fundData.toString().contains("类型")) {
                    String type =  fundData.$("span").regex("(>)(.*)(<)", 2).toString();
                    fund.setType(type);
                } else if (fundData.toString().contains("管理人")) {
                    companyCode =  fundData.links()
                            .regex("(^http://fund.eastmoney.com/company/)([0-9]*)(.html$)", 2).toString();
                }
            }
        } catch (Exception e) {
            logger.error("parse Company info error ", e);
        }
        if (fundDatas != null && fundDatas.size() == 0) {
            throw new NoFundException("fundCode " + fundCode + " is not exist");
        }
        if (companyCode == null) {
            throw new Exception("fundCode: " + fundCode + " companyCode is null");
        }

        if (managerCode == null) {
            throw new Exception("fundCode: " + fundCode + " managerCode is null");
        }
        try {
            spiderCallback.onSaveFund(fund);
            spiderCallback.onSaveFundEstablishDate(fundCode, establishDate);
            spiderCallback.onSavePersonFundRelation(managerCode, fundCode);
            spiderCallback.onSaveCompanyFundRelation(companyCode, fundCode);
        } catch (Exception e) {
            logger.error(" fundCode: " + fundCode, e);
        }



    }

    public static void main(String[] args) {
//        String s = "<tr>\n" +
//                " <td>1</td>\n" +
//                " <td><a href=\"//quote.eastmoney.com/sh601966.html\">601966</a></td>\n" +
//                " <td class=\"tol\"><a href=\"//quote.eastmoney.com/sh601966.html\">玲珑轮胎</a></td>\n" +
//                " <td class=\"tor\"><span id=\"dq601966\"></span></td>\n" +
//                " <td class=\"tor\"><span id=\"zd601966\"></span></td>\n" +
//                " <td class=\"xglj\"><a href=\"ccbdxq_000005_601966.html\" class=\"red\">变动详情</a><a href=\"//fund.eastmoney.com/ba/topic,601966.html\">股吧</a><a href=\"//quote.eastmoney.com/sh601966.html\">行情</a><a href=\"//emweb.eastmoney.com/PC_HSF10/OperationsRequired/Index?type=web&amp;code=sh601966\" style=\"display:none\">档案</a></td>\n" +
//                " <td class=\"tor\">0.51%</td>\n" +
//                " <td class=\"tor\">0.67</td>\n" +
//                " <td class=\"tor\">23.48</td>\n" +
//                "</tr>";
//        String s1 = "<td class=\"tor\"><span id=\"dq601966\"></span></td>";
//        String s2 = "<td class=\"tor\">23.48</td>";
//        String s3 = "<td><a href=\"//quote.eastmoney.com/sh601966.html\">601966</a></td>";
//        Pattern pattern = Pattern.compile(">(((?!span|a).)+)</td>");
//        Matcher matcher = pattern.matcher(s2);
//
//        if (matcher.find()) {
//            String group = matcher.group(1);
//            System.out.println(matcher.group(1));
//        }


    }

}
